﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reactive.Threading.Tasks;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows.Markup;

using alib.Character;
using alib.Enumerable;
using alib.Reflection;
using alib.Debugging;

using agree.schema;

namespace agree
{
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	public enum TdlType
	{
		Type = 0,
		SyntaxRule = 1,
		LexicalRule = 2,
		LexicalEntry = 3,
		GenericLexicalEntry = 4,
		StartSymbol = 5,
		NodeLabel = 6,
		TriggerRule = 7,
		TokenMappingRule = 9,
		LexicalFilteringRule = 10,
		TransferRule = 11,
	};

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///
	///
	///
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[Description("Collection of letter groupings for TDL morphology")]
	[DisplayName("TDL letter set dictionary")]
	public sealed class TdlLetterSets : IdentDict<Char, LetterSet>
	{
		public TdlLetterSets(IIdentity prv)
			: base(prv, 0, CharComparison.IgnoreCase)
		{
		}
	};

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///
	///
	///
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[Description("Letter grouping for TDL morphology")]
	[DisplayName("TDL Letter Set")]
	[ImmutableObject(true)]
	[DictionaryKeyProperty("Identifier"), ContentProperty("Letters"), RuntimeNameProperty("Name")]
	public sealed class LetterSet : identity_base
	{
		public LetterSet(IIdentity prv)
			: base(prv)
		{
		}
		public LetterSet(IIdentity prv, Char ch, String set)
			: this(prv)
		{
			this.Identifier = ch;
			this.Letters = set;
		}

		Char ch;
		public Char Identifier
		{
			get { return this.ch; }
			set { this.ch = Char.ToLowerInvariant(value); }
		}

		String set;
		public String Letters
		{
			get { return this.set; }
			set
			{
				String s = value.Replace(@"\", @"\\").Replace(@"]", @"\]").Replace(@"-", @"\-").Replace(@"^", @"\^");
				this.set = String.Format("([{0}])", s);
			}
		}

		public override String ToString()
		{
			return String.Format("{0} {1}", ch, set);
		}
	};

	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///
	///
	///
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[Description("A collection of related TDL resources")]
	[DisplayName("TDL group")]
	[ContentProperty("Items")]
	public sealed class TdlGroup : identity_named, IIdentList<Tdl>,
		ρρTdlLetterSets
	{
		public TdlGroup(IIdentity prv)
			: base(prv)
		{
			this._letr_sets = new TdlLetterSets(this);
			this._items = new IdentList<Tdl>(this);
		}
		public TdlGroup(IIdentity prv, IEnumerable<Tdl> items)
			: this(prv)
		{
			_items.AddRange(items);
		}

		///////////////////////////////////////////////////////////////////////
		/// 
		[AccessedThroughProperty("Items")]
		readonly IdentList<Tdl> _items;
		public IdentList<Tdl> Items { get { return this._items; } }
		public Tdl this[int index] { get { return _items[index]; } }
		public int Count { get { return _items.Count; } }
		public IEnumerator<Tdl> GetEnumerator() { return _items.GetEnumerator(); }
		IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); }
		void ICollection.CopyTo(Array array, int index) { ((IList)_items).CopyTo(array, index); }
		bool ICollection.IsSynchronized { get { return false; } }
		Object ICollection.SyncRoot { get { return this; } }
		/// 
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		/// 
		[AccessedThroughProperty("TdlLetterSets")]
		readonly TdlLetterSets _letr_sets;
		public TdlLetterSets TdlLetterSets
		{
			get { return _letr_sets; }
			set
			{
				if (_letr_sets == value)
					return;
				LetterSet ls;
				foreach (var kvp in value)
					if (_letr_sets.TryGetValue(kvp.Key, out ls))
						if (ls.ToString() != value.ToString())
							throw new Exception();
						else
							continue;
					else
						_letr_sets.Add(kvp.Key, kvp.Value);
			}
		}
		/// 
		///////////////////////////////////////////////////////////////////////
	};


	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	/// <summary>
	/// 
	/// </summary>
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[ContentProperty("Sequence")]
	[UsableDuringInitialization(true)]
	public sealed class TdlHashAggregator : SelectMany<Tdl, TdlTok>,
		ρρTdlHash
	{
		public TdlHashAggregator(IRtParent parent, IEnumerable<Tdl> src)
			: base(parent, src)
		{
		}
		public TdlHashAggregator(IRtParent parent, IEnumerable<Tdl> src, IFunctor<Tdl, IActiveObj<TdlTok>> project)
			: base(parent, src, project)
		{
		}
		public TdlHashAggregator(IRtParent parent)
			: base(parent)
		{
		}

		List<TdlTokenizer> sources;

		protected override void monad_hook(IObservable<TdlTok> monad)
		{
			var tdl_tok = monad as TdlTokenizer;
			if (monad != null)
			{
				if (sources == null)
					sources = new List<TdlTokenizer>();
				sources.Add(tdl_tok);
			}
			base.monad_hook(monad);
		}

		///////////////////////////////////////////////////////////////////////
		/// 
		public long TdlHash
		{
			get
			{
				long h = 0;
				if (sources != null)
					foreach (var tt in sources)
						h ^= tt.TdlHash;
				return h;
			}
		}
		/// 
		///////////////////////////////////////////////////////////////////////
	};


	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[ContentProperty("Content")]
	[UsableDuringInitialization(true)]
	public sealed class Tdl : identity_base,
		ρρTdlLetterSets,
		ρρFilename,
		ρρTdlType,
		ρρText
	{
		public Tdl(IIdentity prv)
			: base(prv)
		{
			this.letter_sets = (TdlLetterSets)P.TdlLetterSets.GetValue(this);	/// ??? how can this have a value?
			this.tdl_type = TdlType.Type;
		}
		public Tdl(IIdentity prv, String s_tdl)
			: this(prv)
		{
			this._tdl = s_tdl;
		}


		///////////////////////////////////////////////////////////////////////
		///
		[AccessedThroughProperty("TdlType")]
		TdlType tdl_type;
		public TdlType TdlType
		{
			get { return this.tdl_type; }
			set { this.tdl_type = value; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		///
		[AccessedThroughProperty("TdlLetterSets")]
		TdlLetterSets letter_sets;
		public TdlLetterSets TdlLetterSets
		{
			get { return this.letter_sets; }
			set { this.letter_sets = value; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		///
		[AccessedThroughProperty("Filename")]
		String filename;
		public String Filename
		{
			get { return filename ?? String.Empty; }
			set { filename = value; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		///
		[AccessedThroughProperty("Content")]
		Object _tdl;
		public Object Content
		{
			get { return Text; }
			set { _tdl = value; }
		}
		[DebuggerBrowsable(DebuggerBrowsableState.Never)]
		public String Text
		{
			get
			{
				if (_tdl == null)
					return String.Empty;

				var s_tdl = _tdl as String;
				if (s_tdl != null)
					return s_tdl;

				var rf = _tdl as ReadFile;
				if (rf != null)
				{
					this.filename = rf.Filename;
					s_tdl = language_obj_utils.recover_string_content(rf);
				}

				if (s_tdl == null && filename != null)
					s_tdl = File.ReadAllText(filename);

				return s_tdl ?? String.Empty;
			}
			set { _tdl = value; }
		}
		///
		///////////////////////////////////////////////////////////////////////


		public override String ToString()
		{
			var s = this.Text;	/// (do before the next line, because it might fill in the filename)
			var f = Path.GetFileName(filename);
			if (Filename.Length == 0)
				f = "unknown";
			return String.Format("{0}, {1:#,###} chars", f, s.Length);
		}
	};


	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	///
	///
	///
	///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	[ContentProperty("SourceTdl")]
	public class TdlTokenizer : m_observable<Tdl, TdlTok>,
		ρρAgreeConfig,
		ρρTdlLetterSets,
		ρρSourceTdl,
		ρρTdlHash
	{
		const int tab_setting = 4;
		public static readonly HashSet<Char> ident_stop = new HashSet<Char>("\n\r\t \";[]<>!&,.#:=".ToCharArray());

		public TdlTokenizer(IRtParent parent, Tdl s_tdl)
			: base(parent, s_tdl)
		{
		}

		///////////////////////////////////////////////////////////////////////
		/// 
		configuration.AgreeConfig config;
		public configuration.AgreeConfig AgreeConfig
		{
			get { return this.config; }
			set { this.config = value; }
		}
		/// 
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		/// 
		TdlLetterSets letter_sets;
		public TdlLetterSets TdlLetterSets
		{
			get { return this.letter_sets; }
			set { if (value != null) this.letter_sets = value; }
		}
		/// 
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		/// 
		public Tdl SourceTdl
		{
			get { return this.t0; }
			set { this.t0 = value; }
		}
		/// 
		///////////////////////////////////////////////////////////////////////


		///////////////////////////////////////////////////////////////////////
		/// 
		long tdl_hash;
		public long TdlHash
		{
			get { return this.tdl_hash; }
		}
		/// 
		///////////////////////////////////////////////////////////////////////

		protected override void start(Tdl arg)
		{
			s_tdl = (String)arg.Content;
			s_filename = arg.Filename;
			m_cch = s_tdl.Length;
			tdl_hash = 0;
			go();
			_remove_task();
		}

		String s_tdl;
		String s_filename;
		int line = 1;
		int i_start;
		int col_start = 0;
		int i;
		int m_cch;
		Char ch;

		bool Eof { [MethodImpl(MethodImplOptions.AggressiveInlining), DebuggerStepThrough] get { return i >= m_cch; } }

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// 
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		void ProcessLetterSet(String left, String right)
		{
			if (left[0] != '(' || !right.EndsWith(")"))
				error("Unmatched parentheses in letter set definition: '{0} {1}'.", left, right);

			String macro = left.Substring(1);
			String letter_set = right.Remove(right.Length - 1);

			if (macro.Length != 2 || macro[0] != '!')
				error("Inflection letter set macro symbol must be a single character prefixed with '!'.");

			LetterSet ls2, ls = new LetterSet(letter_sets, macro[1], letter_set);

			if (letter_sets.TryGetValue(ls.Identifier, out ls2))
			{
				if (ls2.Letters != ls.Letters)
					error("Cannot redefine inflection letter set macro symbol '{0}'.", macro);
				return;
			}

			letter_sets.Add(ls.Identifier, ls);
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// Process regular morphology specification. Because lines such as [%prefix (!f (!f)] are permitted, the 
		/// parenthesis handling is messy. This code is also complicated by the fact that we are using system
		/// RegEx, so we must be careful exactly what parts of the morphology specification we escape and how we
		/// interpret TDL escaping.
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		TdlTok ProcessAffix(String s_subrule, bool f_pfx, IEnumerable<String> rgs)
		{
			FilePos fp = new FilePos(i, line, i_start - col_start, s_filename);
			TdlTokMorphology tokm = f_pfx ? (TdlTokMorphology)new TdlTokPrefix(fp, s_subrule) : new TdlTokSuffix(fp, s_subrule);

			/// Process each tuple on the morphology specifcation line
			HashSet<Pairing<String>> seen = new HashSet<Pairing<String>>();
			foreach (var pair in rgs.PairOff())
			{
				if (!seen.Add(pair))
				{
					//SysObj.Instance.TransactionStatus(this, tokm.FilePos, "Warning: duplicate regular morphology pair '{2}' ignored.", pair);
					continue;
				}

				/// To handle [%prefix (!f (!f)], we only strip the mandatory outer parentheses after the tuples have
				/// been paired off.
				if (pair.x[0] != '(' || !pair.y.EndsWith(")"))
					error("Unmatched parentheses in inflectional rule: '{0}'.", s_subrule);

				/// For parsing, the affix is already attached, so the right side of each subrule tuple is the input, and
				/// the left side is the output. Hence the reversal
				String s_inp = pair.y.Remove(pair.y.Length - 1).Replace("*", "");
				String s_out = pair.x.Substring(1).Replace("*", "");

				String fwd_replace;
				Regex fwd_regex = MakeRegexFromSubrule(s_inp, s_out, f_pfx, out fwd_replace);

				String bwd_replace;
				Regex bwd_regex = MakeRegexFromSubrule(s_out, s_inp, f_pfx, out bwd_replace);

				/// 6. Create a RegEx and add it to the Tdl token
				tokm.subrules.Add(new MorphologySubrule
				{
					regex = fwd_regex,
					replace = fwd_replace,
					rev_regex = bwd_regex,
					rev_replace = bwd_replace
				});
			}
			return tokm;
		}

		Regex MakeRegexFromSubrule(String s_inp, String s_out, bool f_pfx, out String replace)
		{
			/// A list of distinct (arbitrary) placeholder characters and their eventual String replacements
			List<KeyValuePair<Char, String>> inserts = new List<KeyValuePair<Char, String>>();

			String start_inp = s_inp, start_out = s_out;

			/// Starting placeholder character
			Char ch_ins = '\uE000';


			/// 1. The index of the first RegEx capture group depends on whether it's a prefix or suffix
			int i_capture = f_pfx ? 1 : 2;

			/// 2. Replace each macro sequence with a unique unicode character taken from the private use area
			int ix_additional_use, ix_bang = 0;
			while ((ix_bang = s_inp.IndexOf('!', ix_bang)) != -1 && ix_bang < s_inp.Length - 1)
			{
				Char macro_char = s_inp[ix_bang + 1];
				String macro = "!" + macro_char;
				if (letter_sets == null)
					error("Macro symbol {0} used in inflectional pair ({1} {2}) must be defined in the same file before using it. There are no macro symbols defined.", macro, start_inp, start_out);

				LetterSet ls;
				if (!letter_sets.TryGetValue(macro_char, out ls))
					error("Macro symbol {0} used in inflectional pair ({1} {2}) was not defined.", macro, start_inp, start_out);

				if (!s_out.Contains(macro_char))
					error("Inflectional pair ({0} {1}) uses macro symbol {2} in the input but not the output.", start_inp, start_out, macro);

				/// Generate a new placeholder for the first occurrence of this macro in what will be the input RegEx
				s_inp = s_inp.Remove(ix_bang, 2).Insert(ix_bang, ch_ins.ToString());
				inserts.Add(new KeyValuePair<Char, String>(ch_ins++, ls.Letters));

				/// Check for additional uses of this macro, still in the *input* RegEx
				ix_additional_use = s_inp.IndexOf(macro);
				if (ix_additional_use != -1)
				{
					do
						s_inp = s_inp.Remove(ix_additional_use, 2).Insert(ix_additional_use, ch_ins.ToString());
					while ((ix_additional_use = s_inp.IndexOf(macro, ix_additional_use)) != -1);
					inserts.Add(new KeyValuePair<Char, String>(ch_ins++, @"\" + i_capture.ToString()));
				}

				/// Reference the corresponding capture in the *replacement* part of the RegEx
				s_out = s_out.Replace(macro, "$" + i_capture.ToString());

				/// Keep track of the capture indexes that the RegEx will generate
				i_capture++;
			}

			/// 3. Now that macros are out of the way, we can escape the non-macro parts for RegEx
			s_inp = Regex.Escape(s_inp);

			/// 4. Now we can replace each placeholder with its corresponding string
			foreach (var kvp in inserts)
				s_inp = s_inp.Replace(kvp.Key.ToString(), kvp.Value);

			/// 5. Complete the RegEx expression by including the non-affixed body
			if (f_pfx)
			{
				s_inp = "^" + s_inp + "(.*)$";
				s_out = s_out + "$" + i_capture.ToString();
			}
			else
			{
				s_inp = "^(.*)" + s_inp + "$";
				s_out = "$1" + s_out;
			}

			replace = s_out;
			return new Regex(s_inp, RegexOptions.Compiled | RegexOptions.Singleline);
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// 
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		TdlTok LineBegin()
		{
			if (s_tdl[i] == '%')
			{
				int i_start = ++i;
				while (!Eof && (ch = s_tdl[i]) != 13 && ch != 10)
					i++;

				String s_subrule = s_tdl.Substring(i_start, i - i_start).Trim();
				if (s_subrule.Length == 0)
					error("Expected a morphology specification after '%'");

				if (s_subrule[0] == '(' && s_subrule[s_subrule.Length - 1] == ')')
					s_subrule = s_subrule.Substring(1, s_subrule.Length - 2);

				/// %(letter-set(!h あう...  (Jacy)
				///          ---^
				String sz = s_subrule;
				if (sz.StartsWith("letter-set") && sz[10] != ' ')
					sz = sz.Insert(10, " ");
				else if ((sz.StartsWith("prefix") || sz.StartsWith("suffix")) && sz[6] != ' ')
					sz = sz.Insert(6, " ");

				String[] rgs = sz
					.Replace(@"\(", "(")
					.Replace(@"\)", ")")
					.Replace(@"\!", "!")
					.Replace(@"\?", "?")
					.Replace(")(", ") (")	/// Jacy, eg.  ... (ぎつい ぎつ)(たるい たる) ...
					.Split(Charset.sp, StringSplitOptions.RemoveEmptyEntries);

				if (rgs.Length < 2)
					error("Syntax error in inflectional rule: '{0}'.", s_subrule);

				switch (rgs[0].ToLower())
				{
					case "letter-set":
						if (rgs.Length != 3)
							error("Invalid inflection letter set definition: {0}.", s_subrule);
						ProcessLetterSet(rgs[1], rgs[2]);
						break;

					case "prefix":
						if (((rgs.Length - 1) & 1) > 0)
							error("Invalid inflectional input/output pairing.");
						this.i_start++;
						return ProcessAffix(s_subrule, true, rgs.Skip(1));

					case "suffix":
						if (((rgs.Length - 1) & 1) > 0)
							error("Invalid inflectional input/output pairing.");
						this.i_start++;
						return ProcessAffix(s_subrule, false, rgs.Skip(1));

					default:
						throw error("Unrecognized inflectional rule type: '{0}'.", rgs[0].ToLower());
				}
			}
			return null;
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// 
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		TdlTok LineEnd()
		{
			if (!Eof && ch == 13 && s_tdl[i] == 10)	// windows cr-lf
				i++;
			col_start = i;
			line++;
			return !Eof ? LineBegin() : null;
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// 
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		TdlTok MakeTok(TdlTok.Type t, String form = null)
		{
			return new TdlTok(new FilePos(i_start, line, i_start - col_start, s_filename), t, form);
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// 
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		unsafe void go()
		{
			int sq_nest = 0;
			int ang_nest = 0;
			int dl_nest = 0;
			i = 0;
			long hash;
			byte* hashbytes = (byte*)&hash;

			TdlTok tokm = LineBegin();
			if (tokm != null)
				_item(tokm);

			while (!Eof)
			{
				this.i_start = i;

				hashbytes[i & 7] ^= (byte)(ch = s_tdl[i]);

				if (ch == 13 || ch == 10)
				{
					i++;
					if ((tokm = LineEnd()) != null)
						_item(tokm);
				}
				else if (ch == ' ')		// ignore whitespace
				{
					i++;
				}
				else if (ch == '\t')	// ignore tab
				{
					i++;
					col_start -= tab_setting;
				}
				else if (ch == ';')		// comment extends to end of line
				{
					while (!Eof && (ch = s_tdl[i]) != 13 && ch != 10)
						i++;
					i++;
					if ((tokm = LineEnd()) != null)
						_item(tokm);
				}
				else if (ch == '.')
				{
					i++;
					if (i + 1 < s_tdl.Length && s_tdl[i] == '.' && s_tdl[i + 1] == '.')
					{
						_item(MakeTok(TdlTok.Type.Ellipsis));
						i += 2;
					}
					else
						_item(MakeTok(TdlTok.Type.Dot));
				}
				else if (ch == '[')
				{
					sq_nest++;
					_item(MakeTok(TdlTok.Type.SquareOpen));
					i++;
				}
				else if (ch == ']')
				{
					if (sq_nest == 0)
						error("Unmatched closing square-bracket ']'.");
					_item(MakeTok(TdlTok.Type.SquareClose));
					i++;
					sq_nest--;
				}
				else if (ch == '(')
				{
					i++;
					while (!Eof)
					{
						ch = s_tdl[i++];
						if (ch == ')')
							goto found;
						if (ch < '0' || ch > '9')
							error("Invalid integer value.");
					}
					if (Eof)
						error("End-of-file reading string constant.");
				found:
					_item(MakeTok(TdlTok.Type.Integer, s_tdl.Substring(i_start + 1, i - 2 - i_start)));
				}
				else if (ch == '\"')	// double-quoted "string"
				{
					i++;
					int i_multiline = col_start;
					while (!Eof)
					{
						ch = s_tdl[i++];
						if (ch == '\"')
							break;
						if (ch == 13 || ch == 10)
							if ((tokm = LineEnd()) != null)
								_item(tokm);
					}
					if (Eof)
						error("End-of-file reading string constant.");

					String s = s_tdl.Substring(i_start + 1, i - 2 - i_start);
					col_start = i_multiline;
					if (s.Contains(TdlTok.ReplaceToken))
						_item(MakeTok(TdlTok.Type.RegExRepl, s));
					else
						_item(MakeTok(TdlTok.Type.String, s));
				}
				else if (ch == '\'')	// lisp-style 'string
				{
					i++;
					while (!Eof)
					{
						ch = s_tdl[i];
						if (ident_stop.Contains(ch))		/// ... CAT.HEAD.KEYS.KEY 'temp_loc]]]],  (Jacy) 
							break;
						i++;
						if (ch == 13 || ch == 10)
							if ((tokm = LineEnd()) != null)
								_item(tokm);
					}
					if (Eof)
						error("End-of-file reading lisp-style string constant.");

					_item(MakeTok(TdlTok.Type.String, s_tdl.Substring(i_start + 1, i - 1 - i_start)));
				}
				else if (ch == ':')
				{
					do
						i++;
					while (!Eof && (ch = s_tdl[i]) == ' ');

					if (ch == '=' || ch == '<')
						_item(MakeTok(TdlTok.Type.Define));
					else if (ch == '+')
						_item(MakeTok(TdlTok.Type.Append));
					else
						error_next("Error: ':" + ch + "'; expected :=, :<, or :+");
					i++;
				}
				else if (ch == '<')
				{
					do
						i++;
					while (!Eof && (ch = s_tdl[i]) == ' ');

					if (ch == '!')
					{
						dl_nest++;
						_item(MakeTok(TdlTok.Type.DifferenceListOpen));
						i++;
					}
					else
					{
						ang_nest++;
						_item(MakeTok(TdlTok.Type.AngleOpen));
					}
				}
				else if (ch == '>')
				{
					if (ang_nest == 0)
						error("Unmatched closing angle-bracket '>'.");
					_item(MakeTok(TdlTok.Type.AngleClose));
					i++;
					ang_nest--;
				}
				else if (ch == '!')
				{
					do
						i++;
					while (!Eof && (ch = s_tdl[i]) == ' ');

					if (ch != '>')
						error_next("Error: '!" + ch + "'");
					if (dl_nest == 0)
						error("Unmatched difference list closing symbol '!>'.");

					_item(MakeTok(TdlTok.Type.DifferenceListClose));
					i++;
					dl_nest--;
				}
				else if (ch == '&')
				{
					_item(MakeTok(TdlTok.Type.Ampersand));
					i++;
				}
				else if (ch == ',')
				{
					_item(MakeTok(TdlTok.Type.Comma));
					i++;
				}
				else if (ch == '#' && i + 1 < s_tdl.Length && s_tdl[i + 1] == '|')		// block comment
				{
					i_start = i += 2;
					while (i < s_tdl.Length - 1 && ((ch = s_tdl[i]) != '|' || s_tdl[i + 1] != '#'))
					{
						i++;
						if (ch == 13 || ch == 10)
							if ((tokm = LineEnd()) != null)
								_item(tokm);
					}
					//_item(MakeTok(TdlTok.Type.Comment, s_tdl.Substring(i_start, i - i_start)));
					i++;
					if (Eof)
						error("End-of-file in block comment.");
					i++;
				}
				else if (ch == '#')
				{
					i++;
					while (++i < s_tdl.Length && !ident_stop.Contains(ch = s_tdl[i]))
						;

					if (i_start == i)
						error_next("Unexpected character: " + ch);

					_item(MakeTok(TdlTok.Type.Tag, s_tdl.Substring(i_start, i - i_start).ToLower()));
				}
				else if (ch == '@')
				{
					i++;
					while (++i < s_tdl.Length && !ident_stop.Contains(ch = s_tdl[i]))
						;

					if (i_start == i)
						error_next("Unexpected character: " + ch);

					_item(MakeTok(TdlTok.Type.XMapTag, s_tdl.Substring(i_start, i - i_start).ToLower()));
				}
				else if (ch == '^')	// regex
				{

					_item(MakeTok(TdlTok.Type.RegEx, gather_tmr_regex()));
				}
				else				// identifier
				{
					while (++i < s_tdl.Length && !ident_stop.Contains(ch = s_tdl[i]))
						;

					if (i_start == i)
						error_next("Unexpected character: " + ch);

					var s_ident = s_tdl.Substring(i_start, i - i_start).ToLower();

					_item(MakeTok(TdlTok.Type.Identifier, s_ident));
				}
			}
			this.tdl_hash = hash;
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// <summary>
		/// Gather from [ ^ ...to... $ ], but don't terminate within certain nested occurrences of '$'
		/// </summary>
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		String gather_tmr_regex()
		{
			Debug.Assert(ch == '^');
			bool f_in_charset = false;
			bool f_capture = false;
			String s_ident = "";
			while (++i < s_tdl.Length)
			{
				ch = s_tdl[i];
				if (ch == '\\' && (i + 1) < s_tdl.Length)
				{
					i++;
				}
				else if (f_in_charset)
				{
					if (ch == ']')
						f_in_charset = false;
					else
					{
						String repl = null;
						switch (s_tdl.Substring(i, 9))
						{
							case "[:punct:]":
								repl = @"\p{P}\p{S}";
								break;
							case "[:digit:]":
								repl = @"\p{Nd}";
								break;
							case "[:alpha:]":
								repl = @"\p{L}";
								break;
							case "[:alnum:]":
								repl = @"\p{L}\p{Nd}";
								break;
							case "[:upper:]":
								repl = @"\p{Lu}";
								break;
							case "[:lower:]":
								repl = @"\p{Ll}";
								break;
							default:
								f_in_charset = true;
								continue;
						}
						s_ident += s_tdl.Substring(i_start, i - i_start) + repl;
						i += 8;
						i_start = i + 1;
					}
				}
				else if (ch == '[')
					f_in_charset = true;
				else if (!f_capture)
				{
					if (ch == '$' && !f_in_charset)
					{
						i++;
						s_ident += s_tdl.Substring(i_start, i - i_start);
						return s_ident;
					}
					else if (ch == '(')
						f_capture = true;
				}
				else if (ch == ')')
					f_capture = false;
			}
			throw error("Expected '$'");
		}

		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// instance helpers
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		[DebuggerHidden]
		public Exception error(String fmt, params Object[] args)
		{
			throw ErrorExit(new FilePos(i, line, i - col_start, s_filename), fmt, args);
		}
		[DebuggerHidden]
		public Exception error_next(String fmt, params Object[] args)
		{
			throw ErrorExit(new FilePos(i, line, i + 1 - col_start, s_filename), fmt, args);
		}
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		/// static helpers
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		[DebuggerHidden]
		public static Exception ErrorExit(String msg)
		{
			throw new TdlException("Error: {0}", msg);
		}
		[DebuggerHidden]
		public static Exception ErrorExit(FilePos t, String msg)
		{
			var s = String.Format("file {0}, line {1}, col {2}", t.file, t.line, t.col);
			if (msg != null)
				s = s + " : " + msg;
			throw ErrorExit(s);
		}
		[DebuggerHidden]
		public static Exception ErrorExit(FilePos t, String fmt, params Object[] args)
		{
			throw ErrorExit(t, String.Format(fmt, args));
		}
		[DebuggerHidden]
		public static Exception ErrorExit(TdlTok tt, String fmt, params Object[] args)
		{
			throw ErrorExit(tt.FilePos, fmt, args);
		}
	};
}
